﻿using System.Globalization;
using System.IO;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Windows;

namespace Projekt
{
    class Extractor
    {
        /// <summary>
        /// Method to extract plain text from a webpage 
        /// </summary>
        /// <param name="webpageData">Webpagedata - Object of data holding class WebpageData</param>
        /// <returns>bool - true if extraction was succesful</returns>
        public bool loadWebPage(WebpageData webpageData)
        {
            // Check if prefix http:// is given
            CompareInfo cmpUrl = CultureInfo.InvariantCulture.CompareInfo;
            if (cmpUrl.IsPrefix(webpageData.URL, "http://") == false)
            {
                webpageData.URL = "http://" + webpageData.URL;
            }
                                          
            // Try to extract text from webpage
            try
            {                    
                // Extract HTML-Code from Webpage
                webpageData.HTMLCode = extractHTMLCode(webpageData.URL);                
                MessageBox.Show("Die Webseite wurde erfolgreiche geladen!", "Information", MessageBoxButton.OK, MessageBoxImage.Information);
                return true;
            }
            // If webpage is not valid or available throw exception and show message to user
            catch (System.Exception error)
            {
                if (error is WebException)
                {
                    MessageBox.Show("Es ist ein Fehler aufgetreten!\n" + error.Message
                        + "\n\nBitte überprüfen Sie die URL-Adresse und versuchen Sie es noch einmal!", "Error", MessageBoxButton.OK, MessageBoxImage.Error);
                }
                else if (error is System.ArgumentException)
                {
                    MessageBox.Show("Die eingegebene Adresse war nicht gültig.\n" + error.Message
                        + "\n\nBitte überprüfen Sie die URL-Adresse und versuchen Sie es noch einmal!", "Warnung", MessageBoxButton.OK, MessageBoxImage.Warning);    
                }               
            }
            return false;
        }

        /// <summary>
        /// Method extracts plain Text from HTML-Code
        /// </summary>
        /// <param name="webpageData">Webpagedata - Object of data holding class WebpageData</param>
        public void extractTextFromHTMLCode(WebpageData webpageData)
        {
            // Extract Title from HTML-Code
            webpageData.Title = extractTitle(webpageData.HTMLCode);
            // Extract Content from HTML-Code
            webpageData.Content = extractContent(webpageData.HTMLCode);
            // Remove all JavaScript-Code
            webpageData.Content = removeJavaScriptCode(webpageData.Content);
            // remove CSS-Style Code
            webpageData.Content = removeCSSCode(webpageData.Content);
            // remove HTML-Tags from HTML-Code
            webpageData.PlainText = removeHTMLTags(webpageData.Content);
            // Remove useless whitespaces and linebreaks
            webpageData.PlainText = removeWhitespaces(webpageData.PlainText);
            // Replace special characters with the original
            webpageData.PlainText = replaceSpecialCharacters(webpageData.PlainText);
            // Remove addition Code
            webpageData.PlainText = removeAdditionalCode(webpageData.PlainText);
            
            // create Resultwindow and fill with results of extraction
            ResultWindow resultwindow = new ResultWindow();
            resultwindow.setWebpageTitle(webpageData.Title);
            resultwindow.setWebpageContent(webpageData.PlainText);            
            // show Resultwindow
            resultwindow.Show();
        }
                        
        /// <summary>
        /// Method extracts headlines from HTML-Code
        /// </summary>
        /// <param name="webpageData">Webpagedata - Object of data holding class WebpageData</param>    
        public void extractHeadlinesFromHTMLCode(WebpageData webpageData)
        {
            // Create StringBuilder to save headlines
            StringBuilder headlinesCleaned = new StringBuilder();
            Regex regexTitle = new Regex(@"<h[0-9]+.*?>(.|\n|\r\n)*?</h[0-9]+>");
            MatchCollection matches = regexTitle.Matches(webpageData.HTMLCode);
            foreach (Match match in matches)
            {
                // Get complete content of <h*> tag
                string headlineComplete = match.Groups[0].ToString();
                // Remove all kind of tags to get pure Text
                Regex allTags = new Regex(@"<[^<]+?>");
                string headlineNoTags = allTags.Replace(headlineComplete, "");                
                // Replace special characters
                string headlineCleaned = replaceSpecialCharacters(headlineNoTags);
                headlinesCleaned.Append(headlineCleaned + "\n\n");
            }
            // Remove unnessessary whitespaces
            string headlines = removeWhitespaces(headlinesCleaned.ToString());
            webpageData.Headlines = headlines;
            
            // create Resultwindow and fill with results of extraction
            ResultWindow resultwindow = new ResultWindow();
            resultwindow.setWebpageTitle(webpageData.Title + " -- Headlines");
            resultwindow.setWebpageContent(webpageData.Headlines);
            // show Resultwindow
            resultwindow.Show();
        }

        /// <summary>
        /// Method to show HTML-Code 
        /// </summary>
        /// <param name="webpageData">Webpagedata - Object of data holding class WebpageData</param>
        public void showHTMLCode(WebpageData webpageData)
        {
            // create Resultwindow and fill with results of extraction
            ResultWindow resultwindow = new ResultWindow();
            resultwindow.setWebpageTitle(webpageData.Title + " -- Sourcecode");
            resultwindow.setWebpageContent(webpageData.HTMLCode);
            // show Resultwindow
            resultwindow.Show();
        }


        /// <summary>
        /// Method extracts HTML-Code from webpage
        /// </summary>
        /// <param name="url">string - URL of the webpage</param>
        /// <returns>string - HTML-Code of the webpage</returns>
        private string extractHTMLCode(string url)
        {
            WebClient client = new WebClient();
            Encoding srcEncoding;

            // Get HTML-Code
            byte[] source = client.DownloadData(url);

            System.Text.Encoding enc = System.Text.Encoding.UTF8;
            string sourceString = enc.GetString(source);
            
            // Try matching charset (HTML)
            string encoding = Regex.Match(sourceString, @"charset=([\w-]*)").Groups[1].ToString();

            if (encoding == "")
            {
                // Try matching encoding (XHTML)
                encoding = Regex.Match(sourceString, @"encoding=""([\w-]*)""").Groups[1].ToString();

                if (encoding != "")
                {
                    // If codec is given
                    srcEncoding = Encoding.GetEncoding(encoding);
                }
                else
                {
                    // Else take HTML standard encoding
                    srcEncoding = Encoding.GetEncoding("UTF-8");
                }
            }            
            else
            {
                // If codec is given
                srcEncoding = Encoding.GetEncoding(encoding);
            }

            byte[] resultData = UTF8Encoding.Convert(srcEncoding, UTF8Encoding.UTF8, source);
            string result = UTF8Encoding.UTF8.GetString(resultData);

            return result;
        }
        
        
        /// <summary>
        /// Method extracts Titel from HTML-Code 
        /// </summary>
        /// <param name="code">string - HTML-Code</param>
        /// <returns>string - Title</returns>
        private string extractTitle(string code)
        {
            Regex regexTitle = new Regex("<title>(.*)</title>");
            Match match = regexTitle.Match(code);
            return match.Groups[1].ToString();
        }

        /// <summary>
        /// Method extracts text inside of <body> or <feed> tags from html code
        /// </summary>
        /// <param name="code">string - HTML-Code</param>
        /// <returns>string - Content from HTML-Code</returns>
        private string extractContent(string code)
        {
            Regex regexContent = new Regex("<body.*?>([\\w\\W]*)(</body>){0,1}");
            Match match = regexContent.Match(code);

            string matchedText = match.Groups[1].ToString();

            if (matchedText == "")
            {
                regexContent = new Regex("<feed.*?>([\\w\\W]*)(</feed>){0,1}");

                match = regexContent.Match(code);
            }

            return match.Groups[1].ToString();
        }

        
        /// <summary>
        /// Method to remove all JavaScript-Code
        /// </summary>
        /// <param name="text">string - HTML-Code</param>
        /// <returns>string - HTML-Code without Javascript</returns>
        private string removeJavaScriptCode(string text)
        {
            // JavaScript Regex
            Regex regexJSCode = new Regex(@"(?:<!--[\S\s]*?-->)|(?:<script.*>(.|\n|\r|\r\n)*?</script>)", RegexOptions.IgnoreCase);
            text = regexJSCode.Replace(text, "");

            return text;
        }

        
        /// <summary>
        /// Method to remove all HTML-Tags 
        /// </summary>
        /// <param name="text">string - HTML/Feed-Code</param>
        /// <returns>string - HTML-Code without HTML- & Feed-Tags</returns>
        private string removeHTMLTags(string text)
        {
            // HTML-Tag Regex);
            // regexHTMLTags deletes date of update (<updated>...) in Feeds
            Regex regexHTMLTags = new Regex(@"(<updated>(.*)</updated>)|(<[^<]+?>)");
            text = regexHTMLTags.Replace(text, "");

            return text;
        }

        
        /// <summary>
        /// Method to remove all CSS-Style Code 
        /// </summary>
        /// <param name="text">string - HTML-Code</param>
        /// <returns>string - HTML-Code without CSS</returns>
        private string removeCSSCode(string text)
        {
            Regex regexCSSCode = new Regex(@"(?:<style.*>(.|\n|\r|\r\n)*?</style>)", RegexOptions.IgnoreCase);
            text = regexCSSCode.Replace(text, "");

            return text;
        }

        
        /// <summary>
        /// Method to remove all not needed whitespaces in a text 
        /// </summary>
        /// <param name="text">string - Text to remove whitespaces from</param>
        /// <returns>string - Text without unnecessary whitespaces</returns>
        private string removeWhitespaces(string text)
        {
            // Create regex
            Regex whiteSpaceRegex = new Regex(@"(\s){3,}");
            text = whiteSpaceRegex.Replace(text, "\n\n");
            return text;
        }

               
        /// <summary>
        /// Method to remove additional code, like CDATA Sections
        /// </summary>
        /// <param name="text">string - Text to remove additional code from</param>
        /// <returns>string - cleaned text</returns>
        private string removeAdditionalCode(string text)
        {
            // Create regex
            Regex addCodeRegex = new Regex(@"(/\*)(.|\n)*?(/\*(.|\n)*?(\*/))|(//)(.|\n)*?(//]]>)");
            text = addCodeRegex.Replace(text, "");

            return text;
        }
        
        /*
         * 
         * 
         * @params: string text - Text to replace special characters
         */
        /// <summary>
        /// Method to replace the special characters in HTML-Code with unicode characters
        /// </summary>
        /// <param name="text">string - Text to replace special characters</param>
        /// <returns>string - Text with replaced characters</returns>
        private string replaceSpecialCharacters(string text)
        {
            // Create regex
            Regex specialCharacterRegex = new Regex(@"((?:&nbsp;)|(?:&[Aa]uml;)|(?:&[Uu]uml;)|(?:&[Oo]uml;)|(?:&#[1234567890]+;)|(&amp;)|(&minus;)|(&[rl]s?aquo;)|(&quot;)|(&szlig;))");
            MatchEvaluator specialCharacterEvaluator = delegate(Match match)
            {

                string matchedString = match.Groups[0].ToString();

                switch (matchedString)
                {
                    case "&nbsp;":  return " ";
                    case "&Auml;":  return "Ä";
                    case "&auml;":  return "ä";
                    case "&Uuml;":  return "Ü";
                    case "&uuml;":  return "ü";
                    case "&Ouml;":  return "Ö";
                    case "&ouml;":  return "ö";

                    case "&amp;": return "&";
                    case "&minus;": return "-";
                    case "&laquo;": return "<<";
                    case "&raquo;": return ">>";
                    case "&lsaquo;": return "<";
                    case "&rsaquo;": return ">";
                    case "&quot;": return "\"";
                    case "&szlig;": return "ß";

                    case "&#32;": return " ";
                    case "&#33;": return "!";
                    case "&#34;": return "\"";
                    case "&#35;": return "#";
                    case "&#36;": return "$";
                    case "&#37;": return "%";
                    case "&#38;": return "&";
                    case "&#39;": return "'";
                    case "&#40;": return "(";
                    case "&#41;": return ")";
                    case "&#42;": return "*";
                    case "&#43;": return "+";
                    case "&#44;": return ",";
                    case "&#45;": return "-";
                    case "&#46;": return ".";
                    case "&#47;": return "/";

                    case "&#58;": return ":";
                    case "&#59;": return ";";
                    case "&#60;": return "<";
                    case "&#61;": return "=";
                    case "&#62;": return ">";
                    case "&#63;": return "?";
                    case "&#64;": return "@";

                    case "&#91;": return "[";
                    case "&#92;": return "\\";
                    case "&#93;": return "]";
                    case "&#94;": return "^";
                    case "&#95;": return "_";
                    case "&#96;": return "`";

                    case "&#123;": return "{";
                    case "&#124;": return "|";
                    case "&#125;": return "}";
                    case "&#126;": return "~";

                    case "&#132;": return "„";
                    case "&#147;": return "\"";

                    case "&#161;": return "¡";
                    case "&#162;": return "¢";
                    case "&#163;": return "£";
                    case "&#164;": return "¤";
                    case "&#165;": return "¥";
                    case "&#166;": return "¦";
                    case "&#167;": return "§";
                    case "&#168;": return "¨";
                    case "&#169;": return "©";
                    case "&#170;": return "ª";
                    case "&#171;": return "«";
                    case "&#172;": return "¬";

                    case "&#174;": return "®";
                    case "&#175;": return "¯";
                    case "&#176;": return "°";
                    case "&#177;": return "±";
                    case "&#178;": return "²";
                    case "&#179;": return "³";
                    case "&#180;": return "´";
                    case "&#181;": return "µ";
                    case "&#182;": return "¶";
                    case "&#183;": return "·";
                    case "&#184;": return "¸";
                    case "&#185;": return "¹";
                    case "&#186;": return "º";
                    case "&#187;": return "»";
                    case "&#188;": return "¼";
                    case "&#189;": return "½";
                    case "&#190;": return "¾";

                    case "&#191;": return "¿";
                    case "&#192;": return "À";
                    case "&#193;": return "Á";
                    case "&#194;": return "Â";
                    case "&#195;": return "Ã";
                    case "&#196;": return "Ä";
                    case "&#197;": return "Å";
                    case "&#198;": return "Æ";
                    case "&#199;": return "Ç";
                    case "&#200;": return "È";
                    case "&#201;": return "É";
                    case "&#202;": return "Ê";
                    case "&#203;": return "Ë";
                    case "&#204;": return "Ì";
                    case "&#205;": return "Í";
                    case "&#206;": return "Î";
                    case "&#207;": return "Ï";
                    case "&#208;": return "Ð";
                    case "&#209;": return "Ñ";
                    case "&#210;": return "Ò";
                    case "&#211;": return "Ó";
                    case "&#212;": return "Ô";
                    case "&#213;": return "Õ";
                    case "&#214;": return "Ö";
                    case "&#215;": return "×";
                    case "&#216;": return "Ø";
                    case "&#217;": return "Ù";
                    case "&#218;": return "Ú";
                    case "&#219;": return "Û";
                    case "&#220;": return "Ü";
                    case "&#221;": return "Ý";
                    case "&#222;": return "Þ";
                    case "&#223;": return "ß";
                    case "&#224;": return "à";
                    case "&#225;": return "á";
                    case "&#226;": return "â";
                    case "&#227;": return "ã";
                    case "&#228;": return "ä";
                    case "&#229;": return "å";
                    case "&#230;": return "æ";
                    case "&#231;": return "ç";
                    case "&#232;": return "è";
                    case "&#233;": return "é";
                    case "&#234;": return "ê";
                    case "&#235;": return "ë";
                    case "&#236;": return "ì";
                    case "&#237;": return "í";
                    case "&#238;": return "î";
                    case "&#239;": return "ï";
                    case "&#240;": return "ð";
                    case "&#241;": return "ñ";
                    case "&#242;": return "ò";
                    case "&#243;": return "ó";
                    case "&#244;": return "ô";
                    case "&#245;": return "õ";
                    case "&#246;": return "ö";
                    case "&#247;": return "÷";
                    case "&#248;": return "ø";
                    case "&#249;": return "ù";
                    case "&#250;": return "ú";
                    case "&#251;": return "û";
                    case "&#252;": return "ü";
                    case "&#253;": return "ý";
                    case "&#254;": return "þ";
                    case "&#255;": return "ÿ";

                    default:
                        return "";

                }

            };
            text = specialCharacterRegex.Replace(text, specialCharacterEvaluator);
            return text;

        }
    }
}
